#!/bin/bash
#SBATCH -J GPU_CORRECTNESS_VALIDATION     # 作业名称：GPU正确性验证
#SBATCH -p kshdnormal                     # 队列/分区名称
#SBATCH -N 1                              # 使用1个节点
#SBATCH --gres=dcu:1                      # 申请1个DCU GPU
#SBATCH --ntasks-per-node=1               # 每节点1个任务
#SBATCH --cpus-per-task=8                 # 每个任务使用8个CPU核心
#SBATCH --mem=8G                          # 申请8GB内存
#SBATCH -o logs/runlog/validation_%J.out  # 标准输出日志
#SBATCH -e logs/runlog/validation_%J.err  # 标准错误日志
#SBATCH -t 00:10:00                       # 运行时间限制：10分钟

echo "Start time: `date`"                               # 显示开始时间
echo "SLURM_JOB_ID: $SLURM_JOB_ID"                      # 显示作业号
echo "SLURM_NNODES: $SLURM_NNODES"                      # 显示节点数
echo "SLURM_NTASKS: $SLURM_NTASKS"                      # 显示总任务数
echo "SLURM_TASKS_PER_NODE: $SLURM_TASKS_PER_NODE"      # 显示每节点的任务数
echo "SLURM_CPUS_PER_TASK: $SLURM_CPUS_PER_TASK"        # 显示每个任务使用的CPU数量
echo "SLURM_JOB_PARTITION: $SLURM_JOB_PARTITION"        # 显示队列/分区名称
echo "SLURM_SUBMIT_DIR: $SLURM_SUBMIT_DIR"               # 显示提交作业目录的路径
echo "SLURM_NODELIST: $SLURM_NODELIST"                   # 显示执行节点列表名称

# 创建日志目录
mkdir -p logs/runlog
mkdir -p output/hip

# 设置环境变量
export ROCM_PATH=/opt/rocm
export PATH=$ROCM_PATH/bin:$PATH
export LD_LIBRARY_PATH=$ROCM_PATH/lib:$LD_LIBRARY_PATH

# 显示GPU设备信息
echo "=== GPU Device Information ==="
/opt/rocm/bin/rocminfo | head -20

# 进入HIP源码目录
cd /home/linden/test-stream/src/hip

# 编译HIP验证程序
echo "=== Building HIP Validation Framework ==="
hipcc -o validate_correctness validate_correctness.cpp -std=c++11 -O3

if [ $? -ne 0 ]; then
    echo "❌ Failed to build HIP validation framework"
    exit 1
fi

echo "✅ Successfully built HIP validation framework"

# 运行GPU正确性验证测试
echo "=== Running GPU Correctness Validation ==="
echo "Validating 128 sub-tasks × 1024×1024 matrices on DCU"

# 运行验证测试
./validate_correctness > logs/runlog/validation_results_${SLURM_JOB_ID}.log 2>&1

# 运行分析脚本
echo "=== Analyzing Validation Results ==="
cd /home/linden/test-stream
python3 analyze_validation.py >> logs/runlog/validation_results_${SLURM_JOB_ID}.log 2>&1

# 检查验证结果
if grep -q "VALIDATION PASSED" logs/runlog/validation_results_${SLURM_JOB_ID}.log; then
    echo "✅ GPU Correctness Validation PASSED"
else
    echo "❌ GPU Correctness Validation FAILED"
fi

# 显示结果摘要
echo "=== Validation Summary ==="
echo "Log file: logs/runlog/validation_results_${SLURM_JOB_ID}.log"
echo "Output files: output/hip/*.npy"
echo "Test configuration: 128 sub-tasks × 1,048,576 elements each"
echo "Total elements validated: 134,217,728"

echo "End time: `date`"  # 显示结束时间